#! /usr/bin/env python
# -*- coding: utf-8 -*-   
from scrapy.selector import Selector 
from  logging import info;
from scrapy.spider import BaseSpider as Spider       
from CollectSpider.items import Question,Qa      
from scrapy.http import Request
from CollectSpider.kancloud.dateutil import dateutil
from CollectSpider.kancloud.simhash import simhash
from time import sleep
import time
class SjshdySpider(Spider):  
    name = "sjshdy_spider"  
    #爬虫一次爬取采集的数据
    total_count=0 
    allowed_domains = ["cninfo.com.cn"]  
#     start_urls = [  
#         "http://irm.cninfo.com.cn/ircs/interaction/lastRepliesForSzse.do", 
#     ]
    
#     rules = [ # 定义爬取URL的规则  
#         Rule(sle(allow=("/position.php\?&start=\d{,4}#a")), follow=True, callback='parse_item')  
#     ]  
    #参数化start_urls
    def __init__(self, *args, **kwargs):
        super(SjshdySpider, self).__init__(*args, **kwargs)
        self.task=kwargs
        self.start_urls = [kwargs.get('url_start')]
      
    def parse(self, response): # 提取数据到Items里面，主要用到XPath和CSS选择器提取网页数据  
        sel = Selector(response)  
        #获取最大页码  
        maxNum=int(sel.css('div.PagesBox tr td').xpath("//a[last()-1]/text()").extract()[0].strip())
            #print repr(item).decode("unicode-escape") + '\n' 
            
        #构造请求参数
        #获取隐藏域
        hidden=sel.css("div.PagesBox form").xpath("//input[@type='hidden']")
        params={}
        for inputs in hidden:
            key=inputs.xpath("@name").extract()[0].strip()
            val=inputs.xpath("@value").extract()[0].strip()
            params[key]=val
        
        for num in range(1,2):
            #yield FormRequest.from_response(response,formdata=params,callback = self.parse)
            yield Request(url=(response.url+"?pageNo="+str(num)),callback=self.parse_item,method="POST")
           
        info('parsed ' + str(response))  
        
    def parse_item(self,response):
        sel = Selector(response)  
        #选取内容区域
        oitems=[]
        jitems=[]
        contentDIV = sel.css('div.askBoxOuter,div.answerBoxOuter')
        try:
            i=0
            for msg in contentDIV:  
                i+=1
                if i%2==0:
                    oitem = Question()  
                    oitem['asker'] = msg.css('.userPic span.userName').xpath('text()').extract()[0].strip() if msg.css('.userPic span.userName') else msg.css('.answerBox a.blue2::text').extract()[0].strip()
                    oitem['action']=msg.css('.msgBox span.red1').xpath('text()').extract()[0].strip() if msg.css('.msgBox span.red1') else msg.css('.answerBox span.red1').xpath("text()").extract()[0].strip()
                    oitem['answerer'] =msg.css('.msgBox a.blue2').xpath('text()').extract()[0].strip() if msg.css('.msgBox a.blue2') else msg.css('.answerBox a.blue1::text').extract()[0].strip()
                    oitem['content'] = msg.css('.msgBox a.cntcolor').xpath('text()').extract()[0].strip()  if msg.css('.msgBox a.cntcolor') else msg.css('.answerBox a.cntcolor').xpath("text()").extract()[0].strip()
                    oitem['date'] = msg.css('.msgBox .pubInfo a.date::text').extract()[0].strip()  if msg.css('.msgBox .pubInfo a') else msg.css('.answerBox .pubInfo a.date::text').extract()[0].strip()
                    oitem['source']=u'深交所互动易'+"-->"+response.url
                    oitem['flag']=1 if oitem['action']==u'问' else 2
                    oitems.append(oitem)
                else:
                    jitem = Question()  
                    jitem['asker'] = msg.css('.userPic span.userName').xpath('text()').extract()[0].strip() if msg.css('.userPic span.userName') else msg.css('.answerBox a.blue2::text').extract()[0].strip()
                    jitem['action']=msg.css('.msgBox span.red1').xpath('text()').extract()[0].strip() if msg.css('.msgBox span.red1') else msg.css('.answerBox span.red1').xpath("text()").extract()[0].strip()
                    jitem['answerer'] =msg.css('.msgBox a.blue2').xpath('text()').extract()[0].strip() if msg.css('.msgBox a.blue2') else msg.css('.answerBox a.blue1::text').extract()[0].strip()
                    jitem['content'] = msg.css('.msgBox a.cntcolor').xpath('text()').extract()[0].strip()  if msg.css('.msgBox a.cntcolor') else msg.css('.answerBox a.cntcolor').xpath("text()").extract()[0].strip()
                    jitem['date'] = msg.css('.msgBox .pubInfo a.date::text').extract()[0].strip()  if msg.css('.msgBox .pubInfo a') else msg.css('.answerBox .pubInfo a.date::text').extract()[0].strip()
                    jitem['source']=u'深交所互动易'+"-->"+response.url
                    jitem['flag']=1 if jitem['action']==u'问' else 2
                    jitems.append(jitem)
        
            qaitems=[]
            for i in range(0,len(jitems)):
                qa=Qa()
                qa["time"]=int(time.time())
                qa["time_publish"]=dateutil.get_instance().get_timestramp_by_str(oitems[i]["date"])
                qa["text_content"]=u"【"+jitems[i]["asker"]+u"问:】"+jitems[i]["content"]+u"-->【"+oitems[i]["answerer"]+u"答:】"+oitems[i]["content"]
                qa["url_source"]=jitems[i]["source"]
                qa["task_id"]=self.task["id"]   #任务id
                qa["feature_content"]=str(simhash(jitems[i]["content"].split()))
                qaitems.append(qa)
                sleep(1)   
            #设置采集的成功数
            self.task["count_success"]=len(qaitems)+int(self.task["count_success"])
            self.total_count=len(qaitems)
            #设置失败数
            self.task["count_failed"]=int(self.task["page_count"])-len(qaitems)+int(self.task["count_failed"])
        except Exception,e:
            #设置采集的成功数
            self.task["count_success"]=len(qaitems)+int(self.task["count_success"])
            #设置失败数
            self.task["count_failed"]=int(self.task["page_count"])-len(qaitems)+int(self.task["count_failed"])
            print e
         
        return qaitems

         
    def _process_request(self, request):  
        info('process ' + str(request))  
        return request  
      